/*
 * Javlov - a Java toolkit for reinforcement learning with multi-agent support.
 * 
 * Copyright (c) 2009 Matthijs Snel
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package net.javlov;

/**
 * Agent implementing the Sarsa algorithm.
 * 
 * @author Matthijs Snel
 *
 */
public class SarsaAgent extends TDAgent {

	/**
	 * The Q-value function.
	 */
	protected QFunction q;
	
	/**
	 * The value of the last state-action pair: Q(s_t, a_t).
	 */
	protected double lastQValue;
	
	/**
	 * Flag to learn value function in addition to Q-value function.
	 */
	protected boolean learnValueFunction = false;
	
	/**
	 * Default constructor.
	 */
	public SarsaAgent() {}
	
	/**
	 * Constructs agent with given Q-value function and gamma = 0.9.
	 * @param q the value function to use.
	 */
	public SarsaAgent(QFunction q) {
		this(q, 0.9);
	}
	
	/**
	 * Constructs agent with given Q-value function and gamma.
	 * @param q the value function to use.
	 * @param gamma the discountfactor gamma e [0, 1].
	 */
	public SarsaAgent(QFunction q, double gamma) {
		setQFunction(q);
		setGamma(gamma);
	}
	
	@Override
	public <T> Action doStep( State<T> s, double reward ) {
		if ( learnValueFunction )
			updateValueFunction(s, reward);
		
		double[] qvalues = q.getValues(s);
		Action a = policy.getAction(qvalues);
		double TDerr = reward + gamma*getTargetValue(qvalues, a) - lastQValue;
		q.updatePrevious(TDerr);
		q.setLastAction(a);
		lastQValue = qvalues[a.getID()];
		return a;
	}

	protected double getTargetValue(double[] qvalues, Action a) {
		return qvalues[a.getID()];
	}
	
	@Override
	public <T> Action firstStep( State<T> s ) {
		Action a = super.firstStep(s);
		lastQValue = q.getValue(s, a);
		return a;
	}
	
	@Override
	public void reset() {
		super.reset();
		lastQValue = 0;
		q.reset();
	}
	
	public QFunction getQFunction() {
		return q;
	}

	public void setQFunction(QFunction q) {
		this.q = q;
	}
	
	public boolean isLearnValueFunction() {
		return learnValueFunction;
	}

	public void setLearnValueFunction(boolean learnValueFunction) {
		this.learnValueFunction = learnValueFunction;
	}
}
